// NOTE: This is the old version 1.02 of WSUCONV. It is included here for reference
// purposes. For the latest version, please see ../../tests/wsuconv.cpp.
//---
// wsuconv.cpp - version 1.02
// NCC - Copyright (C) 2012 Willow Schlanger <wschlanger@gmail.com>
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list
//    of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
//    of conditions and the following disclaimer in the documentation and/or other materials
//    provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <cstddef>
#include <cstdio>
#include <string>
#include <sstream>
#include <list>
#include <cctype>

#ifdef _WIN32
// We need GetCommandLineW() on Windows platforms.
#include <windows.h>
#endif

namespace NetwideCompilerCollection
{

typedef unsigned int U4;
typedef signed int S4;

}	// namespace NetwideCompilerCollection

namespace NetwideCompilerCollection
{

enum
{
	NCC_STATUS_EOS = -1,					// end of stream (no error)
	NCC_STATUS_ERROR = -2,					// error condition
	NCC_STATUS__INTERNAL_NEED_DATA = -3		// only used internally
};

namespace NccInternal
{

using namespace std;

class NccRawByteStream
{
public:
	virtual ~NccRawByteStream() { }
	// This returns 0..255, or -1 for EOS (We never return NCC_STATUS_ERROR or -2;
	// if NCC_STATUS_EOS (-1) is returned, see status() to determine whether or not
	// an error has occurred).
	virtual S4 read() = 0;
	// This returns NULL, or an error message set by the last read() call.
	// It is useful to call this after read() returns -1, to make sure we've got
	// a standard EOS condition and not any error (like a disk read error).
	virtual const char *status() = 0;
	virtual void close() = 0;
	// The derived class usually has textTranslationEnabled() return false. If the
	// byte stream is coming from stdin or a file opened in text mode, then you want
	// textTranslationEnabled() to return true. If true is returned by this method,
	// we know not to support UTF-16 inputs since those inputs might contain EOF
	// bytes (0x04 or 0x1a) as part of the UNICODE data; also new-line translation
	// could be active.
	virtual bool textTranslationEnabled() = 0;
};

class NccBufferByteStream :
	public NccRawByteStream
{
	size_t bytesLeft;
	const unsigned char *ptr;
	const char *msg;
public:
	NccBufferByteStream(const unsigned char *ptrT, size_t bufferSizeInBytes)
	{
		ptr = ptrT;
		if(ptr == NULL)
		{
			msg = "NULL input buffer";
			bytesLeft = 0;
		}
		else
		{
			msg = NULL;
			bytesLeft = bufferSizeInBytes;
		}
	}
	
	virtual bool textTranslationEnabled() { return false; }
	
	virtual ~NccBufferByteStream() { }
	
	virtual const char *status() { return msg; }
	
	virtual S4 read()
	{
		if(msg != NULL || ptr == NULL)
		{
			if(msg == NULL && ptr == NULL)
				msg = "attempt to read input buffer after it was closed";
			return NCC_STATUS_EOS;	// return -1
		}
		if(bytesLeft == 0)
			return NCC_STATUS_EOS;	// return -1
		--bytesLeft;
		return static_cast<U4>(*(ptr++));
	}
	
	virtual void close()
	{
		ptr = NULL;
	}
};

class NccCStringByteStream :
	public NccRawByteStream
{
	const char *ptr;
	const char *msg;
public:
	NccCStringByteStream(const char *ptrT)
	{
		ptr = ptrT;
		if(ptr == NULL)
			msg = "NULL input buffer";
		else
			msg = NULL;
	}
	
	virtual bool textTranslationEnabled() { return false; }
	
	virtual ~NccCStringByteStream() { }
	
	virtual const char *status() { return msg; }
	
	virtual S4 read()
	{
		if(msg != NULL || ptr == NULL)
		{
			if(msg == NULL && ptr == NULL)
				msg = "attempt to read input buffer after it was closed";
			return NCC_STATUS_EOS;	// return -1
		}
		if(*ptr == '\0')
			return NCC_STATUS_EOS;	// return -1
		unsigned char tmp = *(ptr++);
		return static_cast<U4>(tmp);
	}
	
	virtual void close()
	{
		ptr = NULL;
	}
};

// The following class is designed to allow reading from files.
// Text mode is designed to work with ASCII-7 or UTF-8 source files, i.e. input from stdin.
// In this mode UTF-16 can't be supported because of the text mode conversions that can take place.
// In text mode, characters 4 and 0x1a are treated like an EOS marker. The user should fail if we're
// in text mode and a byte order mark is encountered (except for the UTF-8 BOM).

class NccCFileByteStream :
	public NccRawByteStream
{
	FILE *file;
	bool textMode;
	const char *msg;
	bool detectedEof;
	bool leaveOpen;
public:
	NccCFileByteStream(FILE *fileT, bool textModeT, bool leaveOpenT /* set to true for e.g. stdin */)
	{
		file = fileT;
		textMode = textModeT;
		leaveOpen = leaveOpenT;
		if(file == NULL)
			msg = "NULL input file";
		else
			msg = NULL;
		detectedEof = false;
	}
	
	virtual ~NccCFileByteStream()
	{
		close();
	}
	
	virtual void close()
	{
		using namespace std;
		if(file != NULL)
		{
			if(!leaveOpen)
				fclose(file);
			file = NULL;
		}
	}
	
	virtual bool textTranslationEnabled() { return textMode; }

	virtual const char *status() { return msg; }
	
	virtual S4 read()
	{
		using namespace std;
		if(msg != NULL)
			return NCC_STATUS_EOS;	// return -1
		if(file == NULL)
		{
			msg = "attempt to read input file after it was closed";
			return NCC_STATUS_EOS;	// return -1
		}
		if(detectedEof)
		{
			return NCC_STATUS_EOS;	// return -1
		}
		int c = fgetc(file);
		
		if(textMode && (c == 4 || c == 0x1a))
		{
			detectedEof = true;
			return NCC_STATUS_EOS;	// return -1
		}
		
		if(c >= 0 & c <= 255)
			return c;
		if(c == EOF && ferror(file) == 0)
		{
			detectedEof = true;
			return NCC_STATUS_EOS;	// return -1
		}
		msg = "error reading from file";
		return NCC_STATUS_EOS;		// return -1
	}
};

}	// namespace NccInternal

using NccInternal::NccRawByteStream;
using NccInternal::NccCStringByteStream;
using NccInternal::NccCFileByteStream;

class NccRawCharacterSource
{
public:
	enum
	{
		FORMAT_ASCII7,   /* source is to consist solely of characters in the range 0..127 inclusive */
		FORMAT_UTF8,     /* this is the most commonly used format--when current() is nonnegative it will be 0..255 */
		FORMAT_UTF16LE,  /* source was little endian. current() always returns native endian results. */
		FORMAT_UTF16BE,  /* source was big endian. current() will still return native endian results. */

		// Note: It is a good idea to not accept UTF-32 from the end user since they could provide characters in the
		// range 0xd800..0xdfff. In general, we want to accept only ASCII-7, UTF-8, or UTF-16 from the end user. Then we
		// want to be the one to convert to UTF-32, as a lot of checking is done when we do that. Then we can convert
		// from UTF-32 to any other format if required, without having to worry about malicious byte sequences.
		FORMAT_UTF32     /* always in native endianness */
	};

	virtual ~NccRawCharacterSource() { }

	// current() returns -1 for EOS, -2 for error (see status() for a message in that case),
	// or it returns the current character in native endianness. If format() is FORMAT_UTF8 then the character will be
	// between 0 and 255 inclusive; FORMAT_ASCII7 means 0 to 127 inclusive; FORMAT_UTF32 means between 0 and 0x10ffff
	// inclusive. For FORMAT_UTF16LE and FORMAT_UTF16BE the value will be between 0 and 0xffff inclusive. Again, native
	// endianness is used so the user need not swap bytes e.g. if FORMAT_UTF16BE is used on a little endian system; we
	// will have already done that. For UTF-8 and UTF-16, the user is responsible for validating the input and doing
	// subsequent reads (via next() / current() calls) when the value returned by current() requires it.
	// See also the definition of NCC_STATUS_EOS and NCC_STATUS_ERROR enum's.
	virtual S4 current() = 0;

	virtual void next() = 0;

	virtual void close() = 0;
	
	// This returns the current format.
	// If neither current() nor next() have been called yet, the file has not been
	// read yet. In that case, format() could cause a file read to get the BOM.
	virtual int format() = 0;
	
	// This returns true if and only if the source starts with a BOM. This is useful,
	// because if the source starts with a BOM in some cases the user may want their
	// output to start with a BOM as well.
	// If neither current() nor next() have been called yet, the file has not been
	// read yet. In that case, sourceHasBom() could cause a file read to get any BOM.
	virtual bool sourceHasBom() = 0;
	
	// This returns "" by default. If current() has returned -2, the meaning is an error
	// condition has been detected. This is usually an error message from the raw byte stream.
	// The user is still responsible for verifying values returned by current() are valid; for
	// example if ASCII7 is used, but current() returns a value in the range 128 to 255 inclusive,
	// the user is responsible for handling that problem.
	virtual std::string status() = 0;

	virtual bool sourceContainsInvalidCharacters()
	{
		return false;
	}
	
	virtual int sourceFormat()
	{
		return format();
	}
};

class NccUtfCharacterSource :
	public NccRawCharacterSource
{
	std::string msg;
	NccRawByteStream &inStream;
	bool bomAllowed;
	int inputMode;
	int inputFormat;
	S4 curChar;		// -1 = EOS, -2 = ERROR (see 'msg'), -3 = initial state (haven't read yet)
	bool inputBomFound;
	unsigned char buf[3];	// big enough for a BOM
	size_t buf_len;
	size_t buf_next;
	bool isTextMode;
	
	void setError(std::string msgT)
	{
		msg = msgT;
		curChar = NCC_STATUS_ERROR;		// -2
	}
public:
	enum
	{
		// With the default input mode, we assume the source is UTF-8 unless it starts with a byte order
		// mark. In that case, we will rely on that mark to tell us the input format. If however the input
		// format is UTF-16 (little or big endian) based on the BOM and the input stream is in text mode,
		// then an error condition is generated. NOTE: if the source is UTF-16 but it nevertheless starts
		// with a UTF-8 BOM, we will wrongly conclude the source is UTF-8. This cannot happen for UTF-16
		// sources that begin with a UTF-16 BOM. The best idea is for source files to be either UTF-8, or
		// to start with a BOM (it is also OK for the source to be UTF-8 and start with a UTF-8 BOM).
		INPUT_MODE_DEFAULT,
		// BOM not allowed (would be invalid ASCII7) -- characters should be in the range 0..127 inclusive.
		INPUT_MODE_ASCII7,
		// Input mode is UTF-8. If we begin with a UTF-8 BOM we ignore it.
		INPUT_MODE_UTF8,
		// Input mode is UTF-8. Any BOM present will be treated as UTF-8 data.
		INPUT_MODE_UTF8_NO_BOM,
		// Input mode is UTF-16 of indicated endianness. Any BOM present will be treated as UTF-16 data.
		// Not compatible with input streams opened in text mode.
		INPUT_MODE_UTF16LE_NO_BOM,
		INPUT_MODE_UTF16BE_NO_BOM,
		// Input mode is UTF-16. If a BOM is absent, assume LE/BE as indicated.
		// If a BOM is present it should be a UTF-16 BOM. The endianness is decided by the BOM in this case and
		// overrides the LE/BE selection made here. Not compatible with input streams opened in text mode. If
		// a UTF-8 BOM is present at the beginning it will be treated like UTF-16 data.
		INPUT_MODE_UTF16_DEFAULT_LE,
		INPUT_MODE_UTF16_DEFAULT_BE
	};
	NccUtfCharacterSource(NccRawByteStream &isT, int inputModeT = INPUT_MODE_DEFAULT) :
		inStream(isT),
		inputMode(inputModeT)
	{
		curChar = NCC_STATUS__INTERNAL_NEED_DATA;	// haven't started yet - do this first before setError() can be called
		inputBomFound = false;
		buf_len = 0;
		buf_next = 0;

		isTextMode = inStream.textTranslationEnabled();
		if(isTextMode)
		{
			if(inputMode == INPUT_MODE_UTF16LE_NO_BOM ||
				inputMode == INPUT_MODE_UTF16BE_NO_BOM ||
				inputMode == INPUT_MODE_UTF16_DEFAULT_LE ||
				inputMode == INPUT_MODE_UTF16_DEFAULT_BE
			)
			{
				setError("UTF-16 input mode requested but input stream is opened in text mode");
			}
		}
		bomAllowed = true;
		if(inputMode == INPUT_MODE_ASCII7 ||
			inputMode == INPUT_MODE_UTF8_NO_BOM ||
			inputMode == INPUT_MODE_UTF16LE_NO_BOM ||
			inputMode == INPUT_MODE_UTF16BE_NO_BOM
		)
		{
			bomAllowed = false;
			
			if(inputMode == INPUT_MODE_ASCII7)
				inputFormat = FORMAT_ASCII7;
			else
			if(inputMode == INPUT_MODE_UTF16LE_NO_BOM)
				inputFormat = FORMAT_UTF16LE;
			else
			if(inputMode == INPUT_MODE_UTF16BE_NO_BOM)
				inputFormat = FORMAT_UTF16BE;
			else
				inputFormat = FORMAT_UTF8;
		}
		else
			inputFormat = -1;	/* current() will set inputFormat based on inputMode and any BOM if allowed */
		
		if(inStream.status() != NULL)
		{
			msg = inStream.status();
			if(!msg.empty())
				curChar = NCC_STATUS_ERROR;
		}
	}

	virtual ~NccUtfCharacterSource()
	{
		close();
	}

	virtual void close()
	{
		inStream.close();
		curChar = NCC_STATUS_EOS;	// -1
	}
	
	// Post: on exit, inputFormat will not be -1 and inputBomFound will be valid.
	virtual S4 current()
	{
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return curChar;		// EOS (-1) or error (-2)
		
		if(curChar == NCC_STATUS__INTERNAL_NEED_DATA)
		{
			// Set default inputFormat if not set yet.
			if(inputFormat == -1)
			{
				if(inputMode == INPUT_MODE_UTF16_DEFAULT_LE)
					inputFormat = FORMAT_UTF16LE;
				else
				if(inputMode == INPUT_MODE_UTF16_DEFAULT_BE)
					inputFormat = FORMAT_UTF16BE;
				else
					inputFormat = FORMAT_UTF8;
			}
			
			if(bomAllowed)
			{
				bomAllowed = false;			// so we don't allow multiple BOMs
				buf_next = buf_len = 0;		// these will already be zero

				// Note. If inStream.read() enters an EOS or error state,
				// it will remain in that state for subsequent reads as well.
				S4 chr = inStream.read();
				if(chr >= 0 && chr <= 255)
				{
					buf[0] = chr;
					buf_len = 1;
				}
				
				if(buf_len == 1)
				{
					if(buf[0] == 0xef || buf[0] == 0xfe || buf[0] == 0xff)
					{
						chr = inStream.read();
						if(chr >= 0 && chr <= 255)
						{
							buf[1] = chr;
							buf_len = 2;
						}
						
						if(buf_len == 2 && buf[0] == 0xef && buf[1] == 0xbb)
						{
							chr = inStream.read();
							if(chr >= 0 && chr <= 255)
							{
								buf[2] = chr;
								buf_len = 3;
							}
						}
						
						if(buf_len == 3)
						{
							if(buf[2] == 0xbf)
							{
								// Got UTF-8 Byte Order Mark. If user specified UTF-16, we did NOT start with a UTF-16 BOM
								// so we're going to keep the contents of the buffer, as if it's valid UTF-16 data bytes!
								// If user did not specify UTF-16, then use the default UTF-8 inputFormat.
								if(inputMode != INPUT_MODE_UTF16_DEFAULT_LE && inputMode != INPUT_MODE_UTF16_DEFAULT_BE)
									buf_next = buf_len = 0;
							}
						}
						else
						if(buf_len == 2)
						{
							if(buf[0] == 0xff && buf[1] == 0xfe)
							{
								// Got UTF-16-LE BOM. If user specified UTF-8, keep the buffer contents, so we can treat it
								// as if it were UTF-8 data bytes (we'll let our user interpret the BOM as UTF-8; let them
								// report any error).
								if(inputMode != INPUT_MODE_UTF8)
								{
									buf_next = buf_len = 0;
									// In case user specified INPUT_MODE_UTF16_DEFAULT_BE -- override input format to LE
									// since we got a BOM.
									inputFormat = FORMAT_UTF16LE;
								}
							}
							else
							if(buf[0] == 0xfe && buf[1] == 0xff)
							{
								// Got UTF-16-BE BOM. If user specified UTF-8, keep the buffer contents, so we can treat it
								// as if it were UTF-8 data bytes (we'll let our user interpret the BOM as UTF-8; let them
								// report any error).
								if(inputMode != INPUT_MODE_UTF8)
								{
									buf_next = buf_len = 0;
									// In case user specified INPUT_MODE_UTF16_DEFAULT_LE -- override input format to BE
									// since we got a BOM.
									inputFormat = FORMAT_UTF16BE;
								}
							}
							if(inputFormat == FORMAT_UTF16LE || inputFormat == FORMAT_UTF16BE)
							{
								if(isTextMode)
								{
									setError("UTF-16 Byte Order Mark detected but input stream is opened in text mode");
								}
							}
						}
					}
				}
			}

			doReadNextByte();

			if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			{
				return curChar;		// EOS (-1) or error (-2)
			}
			
			if(inputFormat == FORMAT_UTF16LE || inputFormat == FORMAT_UTF16BE)
			{
				// Note that we always store characters in native endianness.
				S4 firstByte = curChar;

				doReadNextByte();

				if(curChar == NCC_STATUS_EOS && (inputFormat == FORMAT_UTF16LE || inputFormat == FORMAT_UTF16BE))
				{
					setError("unexpected end of stream while reading UTF-16 data");
				}
				if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
				{
					return curChar;
				}
				
				if(inputFormat == FORMAT_UTF16LE)
					curChar = firstByte | (curChar << 8);
				else
					curChar = curChar | (firstByte << 8);
			}
		}
	
		return curChar;
	}

private:

	void doReadNextByte()
	{
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return;
		
		if(buf_next < buf_len)
		{
			curChar = static_cast<U4>(buf[buf_next++]);
			if(buf_next == buf_len)
				buf_next = buf_len = 0;		// this is not required
			return;
		}

		curChar = inStream.read();
		if(curChar == NCC_STATUS_EOS)
		{
			const char *status_msg = inStream.status();
			if(status_msg != NULL)
				setError(status_msg);
		}
		else
		if(curChar < 0 || curChar > 255)
			setError("unknown error reading from stream");
	}
public:
	
	virtual void next()
	{
		// Make sure we have a current character.
		// This is important and must be done in case a user calls next() a couple of times
		// in a row without prior calls to current().
		if(curChar == NCC_STATUS__INTERNAL_NEED_DATA)
			current();
		
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return;				/* EOS (-1) or error (-2) */
		
		// Invalidate current character so next time we call current() we do an actual read.
		curChar = NCC_STATUS__INTERNAL_NEED_DATA;
	}
	
	// This returns the current format.
	// If neither current() nor next() have been called yet, the file has not been
	// read yet. In that case, format() could cause a file read to get the BOM.
	virtual int format()
	{
		if(inputFormat == -1)
		{
			current();
		}
		return inputFormat;
	}

	virtual bool sourceHasBom()
	{
		current();				// make sure inputBomFound is valid
		return inputBomFound;
	}

	virtual std::string status()
	{
		return msg;
	}
};

// The following adaptor converts a source stream of any format to UTF-32 (in
// the native endian format).
// It checks the input and output. The source must not have a byte order mark.

class NccUtf32CharacterSource :
	public NccRawCharacterSource
{
	NccRawCharacterSource &src;
	bool failOnInvalidCharacters;
	int srcFormat;
	bool gotInvalidCharacter;
	S4 inbuf[4];
	size_t inbuf_next;
	size_t inbuf_len;
	S4 curChar;
	std::string msg;
	bool needSkip;
	bool enableSafeDecodeMode;
	S4 replacementChar;
public:
	NccUtf32CharacterSource(NccRawCharacterSource &srcT, bool failOnInvalidCharactersT,
		bool enableSafeDecodeModeT = true /*set to false to allow e.g. c0 af to decode to '/')*/
	) :
		src(srcT),
		failOnInvalidCharacters(failOnInvalidCharactersT),
		enableSafeDecodeMode(enableSafeDecodeModeT)
	{
		replacementChar = 0xfffd;
		srcFormat = -1;
		gotInvalidCharacter = false;
		inbuf_next = inbuf_len = 0;
		curChar = NCC_STATUS__INTERNAL_NEED_DATA;
		needSkip = false;
		
		msg = src.status();
	}
	
	virtual bool sourceContainsInvalidCharacters()
	{
		return gotInvalidCharacter;
	}

	virtual int sourceFormat()
	{
		if(srcFormat == -1)
			srcFormat = src.sourceFormat();
		return srcFormat;
	}
	
	virtual ~NccUtf32CharacterSource()
	{
	}
	
	virtual int format()
	{
		return NccRawCharacterSource::FORMAT_UTF32;
	}

private:
	S4 getCurrent()
	{
		if(inbuf_len > 0)
			curChar = inbuf[inbuf_next];
		else
			curChar = src.current();
		return curChar;
	}
	
	void skipCurrent()
	{
		if(inbuf_len == 0)
			src.next();
		else
		{
			inbuf_next = (inbuf_next + 1) & 1;
			--inbuf_len;
			if(inbuf_len == 0)
				inbuf_next = 0;
		}
		getCurrent();	// update curChar
	}
	
	S4 peekCurrent(size_t offset)
	{
		if(offset > inbuf_len + 1)
		{
			if(msg.empty())
				msg = "internal error (1) reading UNICODE characters";
			return NCC_STATUS_ERROR;
		}

		if(offset == inbuf_len + 1)
		{
			src.next();
			S4 value = src.current();
			inbuf[(offset + inbuf_next) & 3] = value;
			if(value == NCC_STATUS_ERROR && msg.empty())
			{
				msg = src.status();
				if(msg.empty())
					msg = "error detected reading UNICODE characters";
			}
			++inbuf_len;
		}

		return inbuf[(offset + inbuf_next) & 3];
	}
public:
	
	virtual S4 current()
	{
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return curChar;		// EOS (-1) or error (-2)
		
		if(curChar == NCC_STATUS__INTERNAL_NEED_DATA)
		{
			if(needSkip)
			{
				skipCurrent();
			}
			needSkip = true;
		
			curChar = getCurrent();
			if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			{
				if(curChar == NCC_STATUS_ERROR && msg.empty())
					msg = src.status();
				return curChar;
			}
			
			int fmt = sourceFormat();
			if(fmt == FORMAT_ASCII7)
				processAscii7();
			else
			if(fmt == FORMAT_UTF8)
				processUtf8();
			else
			if(fmt == FORMAT_UTF16LE || fmt == FORMAT_UTF16BE)
				processUtf16();
			// Otherwise we're already UTF-32.
			
			if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
				return curChar;
			
			if(curChar < 0 || curChar > 0x10ffff)
			{
				// We generated an invalid UTF-32 character.
				gotInvalidCharacter = true;
				curChar = replacementChar;
				if(failOnInvalidCharacters)
				{
					curChar = NCC_STATUS_ERROR;
					if(msg.empty())
						msg = "internal error (2) processing UNICODE input";
				}
				return curChar;
			}
			
			// Check that our UTF-32 is valid.
			if((curChar >= 0xd800 && curChar <= 0xdfff) ||
				curChar == 0xfffe || curChar == 0xffff
			)
			{
				// Most likely what happened is the source was UTF-32 and they
				// provided us with bad UTF-32 values.
				gotInvalidCharacter = true;
				curChar = replacementChar;
				if(failOnInvalidCharacters)
				{
					curChar = NCC_STATUS_ERROR;
					if(msg.empty())
						msg = "error processing UNICODE input";
				}
				return curChar;
			}
		}

		return curChar;
	}

private:
	void processAscii7()
	{
		if(curChar >= 128)
		{
			gotInvalidCharacter = true;
			curChar = '?';		// in ASCII7 mode we use a standard question mark as the replacement character
			if(failOnInvalidCharacters)
			{
				if(msg.empty())
				{
					if(curChar <= 255)
					{
						std::stringstream ss;
						ss << std::hex << static_cast<U4>(curChar) << std::dec;
						msg = std::string("expecting standard ASCII characters only, but found character 0x") + ss.str();
					}
					else
						msg = "internal error (3) processing standard ASCII input";
				}
				curChar = NCC_STATUS_ERROR;
			}
		}
	}
	
	void processUtf8()
	{
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return;
		if(curChar < 0 || curChar >= 256)
		{
			gotInvalidCharacter = true;
			curChar = replacementChar;
			if(failOnInvalidCharacters)
			{
				curChar = NCC_STATUS_ERROR;
				if(msg.empty())
					msg = "internal error (4) processing UNICODE input";
			}
			return;
		}
		
		if(curChar == 0xfe || curChar == 0xff)
		{
			gotInvalidCharacter = true;
			curChar = replacementChar;
			if(failOnInvalidCharacters)
			{
				curChar = NCC_STATUS_ERROR;
				if(msg.empty())
				{
					if(curChar == 0xfe)
						msg = "encountered invalid UTF-8 character 0xfe";
					else
						msg = "encountered invalid UTF-8 character 0xff";
				}
			}
			return;
		}
		
		size_t count = 1;
		if(curChar >= 0xfc)
			count = 6;	// illegal
		else
		if(curChar >= 0xf8)
			count = 5;	// illegal
		else
		if(curChar >= 0xf0)
			count = 4;
		else
		if(curChar >= 0xe0)
			count = 3;
		else
		if(curChar >= 0xc0)
			count = 2;
		else
		if(curChar >= 0x80)
		{
			// Unexpected continuation byte.
			gotInvalidCharacter = true;
			if(failOnInvalidCharacters)
			{
				if(msg.empty())
				{
					std::stringstream ss;
					ss << std::hex << static_cast<U4>(curChar) << std::dec;
					msg = std::string("encountered unexpected UTF-8 continuation byte 0x") + ss.str();
				}
				curChar = NCC_STATUS_ERROR;
			}
			else
				curChar = replacementChar;
			return;
		}
		else
			return;		// character is 0..127 inclusive - nothing to do!
		
		if(count >= 5)
		{
			gotInvalidCharacter = true;
			if(failOnInvalidCharacters)
			{
				if(msg.empty())
				{
					std::stringstream ss;
					ss << std::hex << static_cast<U4>(curChar) << std::dec;
					msg = std::string("encountered invalid UTF-8 byte 0x") + ss.str();
				}
				curChar = NCC_STATUS_ERROR;
				return;
			}
			
			// Skip count - 1 bytes of input.
			for(size_t i = 1; i < count; ++i)
			{
				skipCurrent();
				if(getCurrent() < 0x80 || getCurrent() >= 0xc0)
				{
					needSkip = false;
					break;
				}
			}
			
			// If there are 5 characters, we're now pointing at the 5th. Set curChar to the
			// replacement character. User will call next() when they are ready to advance.
			if(curChar != NCC_STATUS_EOS && curChar != NCC_STATUS_ERROR)
				curChar = replacementChar;
			
			return;
		}
		
		unsigned char utfbuf[4];
		utfbuf[0] = curChar;
		
		size_t out_of_range = 0;
		for(size_t i = 1; i < count; ++i)
		{
			// Read ahead.
			S4 tmpChar = peekCurrent(i);
			if(tmpChar == NCC_STATUS_EOS)
			{
				// encountered end of stream in middle of a UTF-8 sequence.
				curChar = replacementChar;
				gotInvalidCharacter = true;
				if(failOnInvalidCharacters)
				{
					curChar = NCC_STATUS_ERROR;
					if(msg.empty())
					{
						msg = "unexpected end of stream while parsing UTF-8 sequence";
					}
				}
				return;
			}
			else
			if(tmpChar < 0 || tmpChar >= 256)
			{
				gotInvalidCharacter = true;
				curChar = NCC_STATUS_ERROR;
				if(msg.empty())
					msg = "internal error (5) reading UNICODE characters";
				return;
			}
			
			utfbuf[i] = tmpChar;
			
			if(tmpChar < 0x80 || tmpChar >= 0xc0)
			{
				out_of_range = i;
				break;
			}
		}
		
		if(out_of_range)
		{
			needSkip = false;
			for(size_t i = 0; i < out_of_range; ++i)
				skipCurrent();
			curChar = replacementChar;
			gotInvalidCharacter = true;
			if(failOnInvalidCharacters)
			{
				curChar = NCC_STATUS_ERROR;
				if(msg.empty())
				{
					msg = "invalid UTF-8 byte sequence encountered";
				}
			}
			return;
		}
		
		// Accept UTF-8 sequence.
		for(size_t i = 1; i < count; ++i)
			skipCurrent();
		if(curChar != NCC_STATUS_EOS && curChar != NCC_STATUS_ERROR)
		{
			curChar = utf8ToUtf32(utfbuf, count);
			// caller will catch invalid UTF-32 values such as the 0xffffffff
			// we use to indicate an internal error (i.e. bad count length).
			// It will also catch a bad UTF-32 value such as 0x110000.
			
			bool overlong = false;
			
			if(curChar >= 0 && curChar <= 0x7f && count != 1)
				overlong = true;
			else
			if(curChar >= 0x80 && curChar <= 0x7ff && count != 2)
				overlong = true;
			else
			if(curChar >= 0x800 && curChar <= 0xffff && count != 3)
				overlong = true;
			
			if(overlong && enableSafeDecodeMode)
			{
				curChar = replacementChar;
				gotInvalidCharacter = true;
				if(failOnInvalidCharacters)
				{
					curChar = NCC_STATUS_ERROR;
					if(msg.empty())
					{
						msg = "found an overlong UTF-8 sequence";
					}
				}
				return;
			}
			
			if((curChar >= 0xd800 && curChar <= 0xdfff) || curChar == 0xffff || curChar == 0xfffe)
			{
				curChar = replacementChar;
				gotInvalidCharacter = true;
				if(failOnInvalidCharacters)
				{
					curChar = NCC_STATUS_ERROR;
					if(msg.empty())
					{
						msg = "invalid UTF-8 sequence encountered";
					}
				}
				return;
			}
		}
	}
	
	void processUtf16()
	{
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return;
		if(curChar < 0 || curChar >= 65536)
		{
			gotInvalidCharacter = true;
			curChar = replacementChar;
			if(failOnInvalidCharacters)
			{
				curChar = NCC_STATUS_ERROR;
				if(msg.empty())
					msg = "internal error (4) processing UNICODE input";
			}
			return;
		}
		
		if(curChar >= 0xd800 && curChar <= 0xdfff)
		{
			S4 initialChar = curChar;
			if(curChar >= 0xdc00)
			{
				gotInvalidCharacter = true;
				if(failOnInvalidCharacters)
				{
					if(msg.empty())
					{
						std::stringstream ss;
						ss << std::hex << static_cast<U4>(curChar) << std::dec;
						msg = std::string("unexpected UTF-16 value 0x") + ss.str();
					}
					curChar = NCC_STATUS_ERROR;
				}
				curChar = replacementChar;
				return;
			}
			U4 first = curChar - 0xd800;
			first = (first << 10) + 0x10000;
			
			S4 second = peekCurrent(1);

			if(second == NCC_STATUS_EOS || second == NCC_STATUS_ERROR)
			{
				if(second == NCC_STATUS_ERROR)
				{
					curChar = NCC_STATUS_ERROR;
					if(msg.empty())
						msg = "error reading UTF-16 value";
					return;
				}
				
				gotInvalidCharacter = true;
				curChar = replacementChar;
				if(failOnInvalidCharacters)
				{
					curChar = NCC_STATUS_ERROR;
					if(msg.empty())
					{
						std::stringstream ss;
						ss << std::hex << static_cast<U4>(initialChar) << std::dec;
						msg = std::string("unexpected end of stream after reading UTF-16 value 0x") + ss.str();
					}
				}
				return;
			}
			
			if(second < 0xdc00 || second > 0xffff)
			{
				gotInvalidCharacter = true;
				curChar = replacementChar;
				if(failOnInvalidCharacters)
				{
					curChar = NCC_STATUS_ERROR;
					if(msg.empty())
						msg = "malformed UTF-16 sequence";
				}
				return;
			}
			
			skipCurrent();
			curChar = first + (second - 0xdc00);
		}
	}

	U4 utf8ToUtf32(unsigned char *utf8Buf, size_t utf8Length)
	{
		if(utf8Length == 1)
			return utf8Buf[0];
		if(utf8Length == 2)
		{
			U4 yyy = (utf8Buf[0] >> 2) & 7;
			U4 yy = utf8Buf[0] & 3;
			U4 xxxxxx = utf8Buf[1] & 0x3f;
			return (yyy << 8) + (yy << 6) + xxxxxx;
		}
		if(utf8Length == 3)
		{
			U4 zzzz = utf8Buf[0] & 0x0f;
			U4 yyyy = (utf8Buf[1] >> 2) & 0x0f;
			U4 yy = utf8Buf[1] & 3;
			U4 xxxxxx = utf8Buf[2] & 0x3f;
			return (zzzz << 12) + (yyyy << 8) + (yy << 6) + xxxxxx;
		}
		if(utf8Length == 4)
		{
			U4 www = utf8Buf[0] & 7;
			U4 zz = (utf8Buf[1] >> 4) & 3;
			U4 zzzz = utf8Buf[1] & 0x0f;
			U4 yyyy = (utf8Buf[2] >> 2) & 0x0f;
			U4 yy = utf8Buf[2] & 3;
			U4 xxxxxx = utf8Buf[3] & 0x3f;
			return (www << 18) + (zz << 16) + (zzzz << 12) + (yyyy << 8) + (yy << 6) + xxxxxx;
		}
		return 0xffffffff;
	}

public:

	virtual void next()
	{
		// Make sure we have a current character.
		// This is important and must be done in case a user calls next() a couple of times
		// in a row without prior calls to current().
		if(curChar == NCC_STATUS__INTERNAL_NEED_DATA)
			current();
		
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return;				// EOS (-1) or error (-2)
		
		// Invalidate current character so next time we call current() we do an actual read.
		curChar = NCC_STATUS__INTERNAL_NEED_DATA;
	}

	virtual void close()
	{
		src.close();
		curChar = NCC_STATUS_EOS;	// -1
	}

	virtual bool sourceHasBom()
	{
		return src.sourceHasBom();
	}

	virtual std::string status()
	{
		return msg;
	}
};

// This class assumes the UTF-32 it's given is well formed, and generates UTF-8 output.
class NccUtf8FromUtf32CharacterSource :
	public NccRawCharacterSource
{
	NccRawCharacterSource &src;
	std::string msg;
	S4 curChar;
	S4 nextChars[4];
	size_t next_char_offset;
	size_t next_char_len;
	bool mustCheckFormat;
	bool needNext;
public:
	NccUtf8FromUtf32CharacterSource(NccRawCharacterSource &srcT) :
		src(srcT)
	{
		msg = src.status();
		curChar = NCC_STATUS__INTERNAL_NEED_DATA;
		next_char_offset = next_char_len = 0;
		
		// We don't want to trigger a read in our constructor, e.g. to
		// detect the input file format from any possible BOM, so we
		// defer the format check until current() is called [note that
		// next() calls current() when necessary].
		mustCheckFormat = true;
		
		needNext = false;
	}
	
	virtual ~NccUtf8FromUtf32CharacterSource()
	{
	}

	virtual S4 current()
	{
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return curChar;
		
		if(mustCheckFormat)
		{
			mustCheckFormat = false;
			if(src.format() != FORMAT_UTF32)
			{
				curChar = NCC_STATUS_ERROR;
				if(msg.empty())
					msg = "invalid input character format";
				return curChar;
			}
		}

		if(curChar == NCC_STATUS__INTERNAL_NEED_DATA)
		{
			next_char_offset = next_char_len = 0;		// this is dead code
			
			if(needNext)
				src.next();
			needNext = true;
			
			S4 tmpChar = src.current();
			
			if(tmpChar <= 0x7f)
			{
				curChar = tmpChar;		// might also be -1 or -2
				
				if(curChar == NCC_STATUS_ERROR)
				{
					if(msg.empty())
					{
						msg = src.status();
						if(msg.empty())
							msg = "encountered an invalid UNICODE character";
					}
				}
			}
			else
			if(tmpChar <= 0x7ff)
			{
				U4 chr = tmpChar;
				unsigned char c1 = 0xc0 + (((chr >> 8) & 7) << 2) + ((chr >> 6) & 3);
				unsigned char c2 = 0x80 + (chr & 0x3f);
				curChar = static_cast<U4>(c1);
				next_char_len = 1;
				nextChars[0] = static_cast<U4>(c2);
			}
			else
			if(tmpChar <= 0xffff)
			{
				U4 chr = tmpChar;
				unsigned char c1 = 0xe0 + ((chr >> 12) & 0x0f);
				unsigned char c2 = 0x80 + (((chr >> 8) & 0x0f) << 2) + ((chr >> 6) & 3);
				unsigned char c3 = 0x80 + (chr & 0x3f);
				curChar = static_cast<U4>(c1);
				next_char_len = 2;
				nextChars[0] = static_cast<U4>(c2);
				nextChars[1] = static_cast<U4>(c3);
			}
			else
			if(tmpChar <= 0x10ffff)
			{
				U4 chr = tmpChar;
				unsigned char c1 = 0xf0 + ((chr >> 18) & 7);
				unsigned char c2 = 0x80 + (((chr >> 16) & 3) << 4) + ((chr >> 12) & 0x0f);
				unsigned char c3 = 0x80 + (((chr >> 8) & 0x0f) << 2) + ((chr >> 6) & 3);
				unsigned char c4 = 0x80 + (chr & 0x3f);
				curChar = static_cast<U4>(c1);
				next_char_len = 3;
				nextChars[0] = static_cast<U4>(c2);
				nextChars[1] = static_cast<U4>(c3);
				nextChars[2] = static_cast<U4>(c4);
			}
			else
			{
				curChar = NCC_STATUS_ERROR;
				if(msg.empty())
					msg = "encountered an invalid UNICODE character";
			}
		}
		
		return curChar;
	}

	virtual void next()
	{
		if(curChar == NCC_STATUS__INTERNAL_NEED_DATA)
			current();
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return;
		
		if(next_char_len == 0)
			curChar = NCC_STATUS__INTERNAL_NEED_DATA;
		else
		{
			curChar = nextChars[next_char_offset++];
			--next_char_len;
			if(next_char_len == 0)
				next_char_offset = 0;
		}
	}

	virtual void close()
	{
		curChar = NCC_STATUS_EOS;
		src.close();
	}
	
	virtual int format()
	{
		return FORMAT_UTF8;
	}
	
	virtual bool sourceHasBom()
	{
		return src.sourceHasBom();
	}
	
	virtual std::string status()
	{
		return msg;
	}

	virtual bool sourceContainsInvalidCharacters()
	{
		return src.sourceContainsInvalidCharacters();
	}
	
	virtual int sourceFormat()
	{
		return src.sourceFormat();
	}
};

// This class assumes the UTF-32 it's given is well formed, and generates UTF-16 output
// in the native endian format, from it.
class NccUtf16FromUtf32CharacterSource :
	public NccRawCharacterSource
{
	NccRawCharacterSource &src;
	std::string msg;
	int currentFormat;
	S4 curChar;
	S4 nextChar;
	bool needNext;
	bool mustCheckFormat;
public:
	NccUtf16FromUtf32CharacterSource(NccRawCharacterSource &srcT) :
		src(srcT)
	{
		msg = src.status();
		
		// Note that the UTF-16 we generate is always in native endianness.
		// Normally format() indicates the SOURCE endianness. Since we're
		// doing translation, we have no knowledge of the source endianness
		// so instad our format() method will return native endianness, which
		// is auto-detected here.
		U4 detect_endian = 0x12345678;
		if(*(unsigned char *)(&detect_endian) == 0x78)
			currentFormat = NccRawCharacterSource::FORMAT_UTF16LE;
		else
			currentFormat = NccRawCharacterSource::FORMAT_UTF16BE;
		
		curChar = NCC_STATUS__INTERNAL_NEED_DATA;
		nextChar = NCC_STATUS__INTERNAL_NEED_DATA;
		needNext = false;
		
		mustCheckFormat = true;
	}
	
	virtual ~NccUtf16FromUtf32CharacterSource()
	{
	}

	virtual S4 current()
	{
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return curChar;
		
		if(mustCheckFormat)
		{
			mustCheckFormat = false;
			if(src.format() != FORMAT_UTF32)
			{
				curChar = NCC_STATUS_ERROR;
				if(msg.empty())
					msg = "invalid input character format";
				return curChar;
			}
		}

		if(curChar == NCC_STATUS__INTERNAL_NEED_DATA)
		{
			if(needNext)
				src.next();
			needNext = true;
			
			S4 tmpChar = src.current();
			
			if(tmpChar == NCC_STATUS_ERROR && msg.empty())
				msg = src.status();
			
			if(tmpChar > 0xffff && tmpChar <= 0x10ffff)
			{
				U4 vp = tmpChar - 0x10000;
				U4 vh = vp >> 10;
				U4 v1 = vp & 0x3ff;
				U4 w1 = 0xd800 + vh;
				U4 w2 = 0xdc00 + v1;
				
				curChar = w1;
				nextChar = w2;
			}
			else
			{
				curChar = tmpChar;
				nextChar = NCC_STATUS__INTERNAL_NEED_DATA;
			}
		}
		
		return curChar;
	}

	virtual void next()
	{
		if(curChar == NCC_STATUS__INTERNAL_NEED_DATA)
			current();
		if(curChar == NCC_STATUS_EOS || curChar == NCC_STATUS_ERROR)
			return;
		curChar = nextChar;		// may be NCC_STATUS__INTERNAL_NEED_DATA (-3)
		nextChar = NCC_STATUS__INTERNAL_NEED_DATA;
	}

	virtual void close()
	{
		if(curChar != NCC_STATUS_ERROR)
			curChar = NCC_STATUS_EOS;
		src.close();
	}
	
	// This returns the current format -- so we know how to interpret the values
	// returned by current(). This is NOT our input's format, which is always UTF-32.
	// See also sourceFormat().
	virtual int format()
	{
		return currentFormat;
	}
	
	virtual bool sourceHasBom()
	{
		return src.sourceHasBom();
	}
	
	virtual std::string status()
	{
		return msg;
	}

	virtual bool sourceContainsInvalidCharacters()
	{
		return src.sourceContainsInvalidCharacters();
	}
	
	virtual int sourceFormat()
	{
		return src.sourceFormat();
	}
};

namespace NccInternal
{

class NccCommonUtils
{
#ifdef _WIN32
	static std::wstring convertUtf8ToUtf16(const char *src)
	{
		std::wstring result;
		
		NccCStringByteStream in1(src);
		NccUtfCharacterSource in2(in1, NccUtfCharacterSource::INPUT_MODE_UTF8_NO_BOM);
		NccUtf32CharacterSource in3(in2, true /* fail for invalid characters */);
		NccUtf16FromUtf32CharacterSource in(in3);
		
		while(in.current() >= 0 && in.current() <= 0xffff)
		{
			result += static_cast<wchar_t>(in.current());
			in.next();
		}
		
		if(in.current() != NCC_STATUS_EOS)
		{
			result.clear();		// error parsing 'src'
		}
		
		return result;
	}
	
	static std::string convertUtf16ToUtf8(const wchar_t *src)
	{
		std::string result;
		
		size_t len = 0;
		for(; src[len] != L'\0'; ++len)
			;
		
		NccBufferByteStream in1((const unsigned char *)(src), len * 2);

		NccUtfCharacterSource in2(in1, NccUtfCharacterSource::INPUT_MODE_UTF16LE_NO_BOM);
		NccUtf32CharacterSource in3(in2, true /* fail for invalid characters */);
		NccUtf8FromUtf32CharacterSource in(in3);
		
		while(in.current() >= 0 && in.current() <= 0xff)
		{
			result += static_cast<char>(in.current());
			in.next();
		}
		
		if(in.current() != NCC_STATUS_EOS)
		{
			result.clear();		// error parsing 'src'
		}
		
		return result;
	}
#endif

public:

	// This function opens the given file with the requested mode.
	// On platforms supporting UTF-8 file names, fopen() itself is
	// used. On Windows, the file name and mode is first converted
	// to UTF-16, then _wfopen() is used -- so we get UNICODE support.
	static FILE *universalOpen(const char *file, const char *mode)
	{
#ifdef _WIN32
		// On Windows, fopen() does not accept UTF-8 filenames.
		// So we need to convert to UTF-16-LE and pass in to _wfopen().
		std::wstring filew = convertUtf8ToUtf16(file);
		std::wstring modew = convertUtf8ToUtf16(mode);
		if(filew.empty() || modew.empty())
			return NULL;	// error parsing the given UTF-8 'file' or 'mode'
		return _wfopen(filew.c_str(), modew.c_str());
#else
		// Assume fopen() accepts UTF-8 filenames!
		return fopen(file, mode);
#endif
	}
	
	static int removeFile(const char *filename)
	{
#ifdef _WIN32
		std::wstring filew = convertUtf8ToUtf16(filename);
		if(filew.empty())
			return -1;	// error parsing the given UTF-8 'filename'.
		return _wremove(filew.c_str());
#else
		return remove(filename);
#endif
	}
	
	// getCommandLine() provides a platform-independent way to get UTF-8 command line arguments.
	// Note: quotation marks are supported. "foo bar" becomes one token, with the quotes removed.
	// The quotes have the effect of allowing spaces in an argument, i.e. for a file name.
	// On UNIX-like systems, we rely on argc and argv[] instead. We remove quotation marks from
	// argv[] lists as well. I trust a command like this: program foo\ bar
	// will on UNIX-like systems produce e.g. {"program", "foo bar"} for argv[].
	static void getCommandLine(std::list<std::string> &result, int argc, const char **argv)
	{
		using namespace std;
		
		result.clear();
#ifdef _WIN32
		LPWSTR cmdLineWin = GetCommandLineW();
		std::string cmdLine = convertUtf16ToUtf8(cmdLineWin);
		
		// Parse the command line.
		std::string::iterator cur = cmdLine.begin();
		std::string::iterator end = cmdLine.end();
		
		while(cur != end)
		{
			// Skip any space characters.
			while(cur != end && isspace(*cur))
				++cur;
			if(cur == end)
				break;
			
			// Check for quote.
			std::string token;
			if(*cur == '\"')
			{
				do
				{
					token += *cur;
					++cur;
					if(cur == end)
						break;
				}	while(*cur != '\"');
				if(cur != end)
				{
					token += *cur;
					++cur;
				}
			}
			else
			{
				do
				{
					token += *cur;
					++cur;
					if(cur == end)
						break;
				}	while(!isspace(*cur));
			}
			
			result.push_back(token);
		}
#else
		for(size_t i = 0; argc > 0; --argc, ++i)
		{
			std::string token = argv[i];
			std::string output;
			for(std::string::iterator i = token.begin(); i != token.end(); ++i)
			{
				if(*i != '\"')
					output += *i;
			}
			result.push_back(output);
		}
#endif
	}
};

}	// namespace NccInternal

using NccInternal::NccCommonUtils;

}	// namespace NetwideCompilerCollection

//=================================================================================================
//=================================================================================================
// Begin main program
//-------------------------------------------------------------------------------------------------

#include <stdexcept>
#include <iostream>

void print_banner()
{
	std::cout << "WSUCONV      Converts between UTF-8, UTF-16, and Standard ASCII" << std::endl;
}

void print_basic_copyright()
{
	std::cout << "Copyright (C) 2012 Willow Schlanger (use --about for more info)\n";
}

void print_usage()
{
	std::cout << "Usage: wsuconv [options] infile outfile\n";
}

void print_help()
{
	print_banner();
	print_basic_copyright();
	std::cout << "\n";
	print_usage();
	std::cout << "\n";
	std::cout << "options:\n";
	std::cout << "--about   print legal information and author contact e-mail(s)\n";
	std::cout << "--help    print this help message\n";
	std::cout << "--omit-bom\n";
	std::cout << "  Normally when we generate UTF outputs, we write a Byte Order Mark.\n";
	std::cout << "  To produce output without a BOM, use this option. Note that if the\n";
	std::cout << "  output format is Standard ASCII, a BOM is never written.\n";
	std::cout << "  NOTE: if the input and output format are both UTF-8, a BOM is written\n";
	std::cout << "        only if present in the source.\n";
	std::cout << "--in=x\n";
	std::cout << "  Normally we auto-detect the input format based on its Byte Order mark\n";
	std::cout << "  and we assume the input is UTF-8 if no such mark is present. You can\n";
	std::cout << "  use this option to force the input format to UTF-16LE or UTF-16BE\n";
	std::cout << "  (useful when the source is in UNICODE but has no Byte Order Mark).\n";
	std::cout << "  x should be one of these: std-ascii utf-16le utf-16be utf-8\n";
	std::cout << "  If --in=utf-8 is used, the input may still begin with a UTF-8 Byte\n";
	std::cout << "  Order Mark. If --in=utf-16le or --in=utf-16be is used and the input\n";
	std::cout << "  begins with a UTF-16 Byte Order Mark, the endianness of the UTF-16\n";
	std::cout << "  will be determined from the Byte Order Mark, thus overriding the\n";
	std::cout << "  'le' or 'be' (little endian or big endian) format specified here.\n";
	std::cout << "--out=x\n";
	std::cout << "  Specify the output format (by default we generate UTF-8 output).\n";
	std::cout << "  x should be one of these: std-ascii utf-16le utf-16be utf-8\n";
	std::cout << "--newline=x\n";
	std::cout << "  This option controls the output line-ending format. The default\n";
	std::cout << "  behavior is to preserve CR and LF characters as found in the input.\n";
	std::cout << "  x should be one of: default lf crlf cr\n";
	std::cout << "--fail-on-errors\n";
	std::cout << "  Normally we ignore most error conditions, such as invalid UNICODE\n";
	std::cout << "  characters. When this happens, a replacement character (U+FFFD)\n";
	std::cout << "  is generated except for Standard ASCII mode where a '?' character\n";
	std::cout << "  is generated. If --fail-on-errors is specified, we will generate\n";
	std::cout << "  an error message, abort, and delete the output file if any errors\n";
	std::cout << "  are found with the input." << std::endl;
}

void print_about()
{
	print_banner();
	std::cout << "Copyright (C) 2012 Willow Schlanger      <wschlanger@gmail.com>\n";
	std::cout << "\n";
	std::cout << "Redistribution and use in source and binary forms, with or without\n";
	std::cout << "modification are permitted provided that the following conditions are met:\n";
	std::cout << "\n";
	std::cout << "1. Redistributions of source code must retain the above copyright notice,\n";
	std::cout << "   this list of conditions and the following disclaimer.\n";
	std::cout << "2. Redistributions in binary form must reproduce the above copyright notice,\n";
	std::cout << "   this list of conditions and the following disclaimer in the documentation\n";
	std::cout << "   and/or other materials provided with the distribution.\n";
	std::cout << "\n";
	std::cout << "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n";
	std::cout << "AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n";
	std::cout << "IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n";
	std::cout << "DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR\n";
	std::cout << "ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n";
	std::cout << "(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n";
	std::cout << "LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON\n";
	std::cout << "ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n";
	std::cout << "(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n";
	std::cout << "SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." << std::endl;
}

// sample function to illustrate how to use the above code.
void write_utf16le_from_utf32(const char *ofname, NetwideCompilerCollection::NccRawCharacterSource &src2, FILE *fo = NULL)
{
	using namespace NetwideCompilerCollection;
	using namespace std;
	
	if(src2.format() != NccRawCharacterSource::FORMAT_UTF32)
		throw std::runtime_error("source is not UTF-32");

	NccUtf16FromUtf32CharacterSource src(src2);

	if(!src2.status().empty())
	{
		throw std::runtime_error(std::string("encountered error: ") + src.status());
	}
	
	if(fo == NULL)
	{
		fo = NccCommonUtils::universalOpen(ofname, "wb");
		if(fo == NULL)
			throw std::runtime_error(std::string("unable to open file for writing: ") + std::string(ofname));
	}
	
	// Write UTF-16-LE Byte Order Mark.
	fputc(0xff, fo);
	fputc(0xfe, fo);

	// While we get valid characters, send them to the output file.
	while(src.current() >= 0 && src.current() <= 0xffff)
	{
		// note: even if src.current() is '\r' or '\n', just copy to output.
		// for the purpose of this sample we don't want to normalize or convert CR/LF's.
		fputc(src.current() & 0xff, fo);
		fputc((src.current() >> 8) & 0xff, fo);
		src.next();
	}
	
	fclose(fo);
	if(src.current() != NCC_STATUS_EOS)
	{
		// -1 means EOS. -2 means ERROR.
		if(src.current() == NCC_STATUS_ERROR && !src.status().empty())
		{
			throw std::runtime_error(std::string("encountered error: ") + src.status());
		}
		throw std::runtime_error("internal error (6)");
	}
}

void write_utf16be_from_utf32(const char *ofname, NetwideCompilerCollection::NccRawCharacterSource &src2)
{
	using namespace NetwideCompilerCollection;
	using namespace std;
	
	if(src2.format() != NccRawCharacterSource::FORMAT_UTF32)
		throw std::runtime_error("source is not UTF-32");

	NccUtf16FromUtf32CharacterSource src(src2);

	if(!src2.status().empty())
	{
		throw std::runtime_error(std::string("encountered error: ") + src.status());
	}
	
	FILE *fo = fopen(ofname, "wb");
	if(fo == NULL)
		throw std::runtime_error(std::string("unable to open file for writing: ") + std::string(ofname));
	
	// Write UTF-16-BE Byte Order Mark.
	fputc(0xfe, fo);
	fputc(0xff, fo);

	// While we get valid characters, send them to the output file.
	while(src.current() >= 0 && src.current() <= 0xffff)
	{
		// note: even if src.current() is '\r' or '\n', just copy to output.
		// for the purpose of this sample we don't want to normalize or convert CR/LF's.
		fputc((src.current() >> 8) & 0xff, fo);
		fputc(src.current() & 0xff, fo);
		src.next();
	}
	
	fclose(fo);
	if(src.current() != NCC_STATUS_EOS)
	{
		// -1 means EOS. -2 means ERROR.
		if(src.current() == NCC_STATUS_ERROR && !src.status().empty())
		{
			throw std::runtime_error(std::string("encountered error: ") + src.status());
		}
		throw std::runtime_error("internal error (6)");
	}
}

void write_utf8_from_utf32(const char *ofname, NetwideCompilerCollection::NccRawCharacterSource &src2)
{
	using namespace NetwideCompilerCollection;
	using namespace std;
	
	if(src2.format() != NccRawCharacterSource::FORMAT_UTF32)
		throw std::runtime_error("source is not UTF-32");

	NccUtf8FromUtf32CharacterSource src(src2);

	if(!src2.status().empty())
	{
		throw std::runtime_error(std::string("encountered error: ") + src.status());
	}
	
	FILE *fo = fopen(ofname, "wb");
	if(fo == NULL)
		throw std::runtime_error(std::string("unable to open file for writing: ") + std::string(ofname));
	
	// Write UTF-8 Byte Order Mark.
	fputc(0xef, fo);
	fputc(0xbb, fo);
	fputc(0xbf, fo);

	// While we get valid characters, send them to the output file.
	while(src.current() >= 0 && src.current() <= 0xff)
	{
		// note: even if src.current() is '\r' or '\n', just copy to output.
		// for the purpose of this sample we don't want to normalize or convert CR/LF's.
		fputc(src.current(), fo);
		src.next();
	}
	
	fclose(fo);
	if(src.current() != NCC_STATUS_EOS)
	{
		// -1 means EOS. -2 means ERROR.
		if(src.current() == NCC_STATUS_ERROR && !src.status().empty())
		{
			throw std::runtime_error(std::string("encountered error: ") + src.status());
		}
		throw std::runtime_error("internal error (6)");
	}
}

#if 0
class Utf32TestGenerator :
	public NetwideCompilerCollection::NccRawCharacterSource
{
	NetwideCompilerCollection::S4 curChar;
public:
	Utf32TestGenerator()
	{
		curChar = 0;
	}

	virtual ~Utf32TestGenerator() { }

	virtual NetwideCompilerCollection::S4 current()
	{
		return curChar;
	}

	virtual void next()
	{
		if(curChar < 0)
			return;
		++curChar;
		if(curChar == 0xd800)
			curChar = 0xe000;
		if(curChar == 0xfffe)
			curChar = 0x10000;
		if(curChar == 0x110000)
			curChar = -1;
	}

	virtual void close()
	{
		curChar = -1;
	}
	
	// This returns the current format.
	// If neither current() nor next() have been called yet, the file has not been
	// read yet. In that case, format() could cause a file read to get the BOM.
	virtual int format()
	{
		return FORMAT_UTF32;
	}
	
	virtual bool sourceHasBom()
	{
		return false;
	}
	
	virtual std::string status()
	{
		return "";
	}
};

void do_test_suite()
{
	using namespace std;
	using namespace NetwideCompilerCollection;
	
	std::cout << "Generating files... " << std::flush;
	{
		Utf32TestGenerator testSource;
		write_utf16le_from_utf32("utf16le_test_out.txt", testSource);
	}
	{
		Utf32TestGenerator testSource;
		write_utf16be_from_utf32("utf16be_test_out.txt", testSource);
	}
	{
		Utf32TestGenerator testSource;
		write_utf8_from_utf32("utf8_test_out.txt", testSource);
	}
	std::cout << "done" << std::endl;
	
	// Decode each file back to UTF-32 and compare.
	const char *filenames[] = {"utf16le_test_out.txt", "utf16be_test_out.txt", "utf8_test_out.txt", NULL};
	for(const char **fn = filenames; *fn != NULL; ++fn)
	{
		std::cout << "\nTranslating " << *fn << "..." << std::endl;
		
		NccCFileByteStream is(fopen(*fn, "rb"), false, false);

		// Create a generic UTF character source that reads from the byte stream.
		NccUtfCharacterSource src1(is);

		// Create a UTF-32 character source that reads from the generic UTF character source.
		// Fail on invalid characters, so we can remove them.
		NccUtf32CharacterSource src2(src1, true /* fail on invalid characters */);
		
		Utf32TestGenerator correct;
		S4 lastGood = -1;
		while(src2.current() >= 0)
		{
			lastGood = src2.current();
			if(lastGood != correct.current())
			{
				std::cout << "Discrepency found!!! Wanted: " << std::hex << correct.current() << std::dec << std::endl;
				break;
			}
			src2.next();
			correct.next();
		}
		std::cout << "Message = [" << src2.status() << "]" << std::endl;
		std::cout << std::hex << lastGood << std::dec << std::endl;
	}
}
#endif

namespace WsuInternal
{

using namespace std;
using namespace NetwideCompilerCollection;

class WsuConverter
{
	bool failOnInvalidChars;
	int out_utf_bits;
	bool out_big_endian;
	bool out_want_bom;
	const char *targetNewline;
	int inputMode;
public:
	WsuConverter()
	{
		failOnInvalidChars = false;
		out_utf_bits = 8;
		out_big_endian = false;
		out_want_bom = true;
		targetNewline = NULL;	// no new-line translation
		inputMode = NccUtfCharacterSource::INPUT_MODE_DEFAULT;
	}
	
	virtual ~WsuConverter()
	{
	}
	
	bool parseOptions(std::list<std::string> &options)
	{
		for(std::list<std::string>::iterator i = options.begin(); i != options.end(); ++i)
		{
			if(*i == "--omit-bom")
				out_want_bom = false;
			else
			if(*i == "--out=std-ascii")
				out_utf_bits = 7;
			else
			if(*i == "--out=utf-16le")
			{
				out_utf_bits = 16;
				out_big_endian = false;
			}
			else
			if(*i == "--out=utf-16be")
			{
				out_utf_bits = 16;
				out_big_endian = true;
			}
			else
			if(*i == "--out=utf-16le")
				out_utf_bits = 8;
			else
			if(*i == "--newline=lf")
				targetNewline = "\n";
			else
			if(*i == "--newline=crlf")
				targetNewline = "\r\n";
			else
			if(*i == "--newline=cr")
				targetNewline = "\r";
			else
			if(*i == "--fail-on-errors")
				failOnInvalidChars = true;
			else
			if(*i == "--in=std-ascii")
				inputMode = NccUtfCharacterSource::INPUT_MODE_ASCII7;
			else
			if(*i == "--in=utf-16le")
				inputMode = NccUtfCharacterSource::INPUT_MODE_UTF16_DEFAULT_LE;
			else
			if(*i == "--in=utf-16be")
				inputMode = NccUtfCharacterSource::INPUT_MODE_UTF16_DEFAULT_BE;
			else
			if(*i == "--in=utf-8")
				inputMode = NccUtfCharacterSource::INPUT_MODE_UTF8;
			else
			{
				std::cout << "ERROR: Invalid command-line argument: " << *i << std::endl;
				return false;
			}
		}
		
		return true;	// success
	}
	
	bool convert(FILE *fi, FILE *fo)
	{
		bool ok = true;

		// Create a byte stream that just reads from a file.
		// We are not responsible for closing the file.

		NccCFileByteStream is(fi, false, true /* leave open when done */);

		// Create a generic UTF character source that reads from the byte stream.
		NccUtfCharacterSource src1(is, inputMode);
		
		// Create a UTF-32 character source that reads from the generic UTF character source.
		NccUtf32CharacterSource src2(src1, failOnInvalidChars);
		
		if(src2.format() != NccRawCharacterSource::FORMAT_UTF32)
			throw std::runtime_error("internal UNICODE translation error");

		NccRawCharacterSource *src = NULL;
		NccRawCharacterSource *freeSrc = NULL;
		
		bool utf8SkipBom = false;
		if(out_utf_bits == 8)
		{
			freeSrc = src = new NccUtf8FromUtf32CharacterSource(src2);
			
			// If the source and output formats are both UTF-8 and the source does not begin with
			// a BOM, do not write a BOM to the output. This allows us to process Standard ASCII
			// files with default options (where we treat the input as UTF-8 and generate UTF-8
			// output). A user would not want a Standard ASCII file to get a UTF-8 BOM injected
			// at the beginning; nor do they want to have to use an option like --omit-bom for
			// this case.
			if(src->sourceFormat() == NccRawCharacterSource::FORMAT_UTF8 && !src->sourceHasBom())
				utf8SkipBom = true;
		}
		else
		if(out_utf_bits == 16)
			freeSrc = src = new NccUtf16FromUtf32CharacterSource(src2);
		else
		{
			// Output is Standard ASCII.
			src = &src2;
		}
		
		try
		{
			// Write any BOM.
			if(!utf8SkipBom)
				write_bom(fo);

			while(src->current() >= 0)
			{
				if(targetNewline != NULL && (src->current() == '\r' || src->current() == '\n'))
				{
					// Get (and count) any '\r' (CR) characters first.
					size_t num_cr = 0;
					while(src->current() == '\r')
					{
						++num_cr;
						src->next();
					}
					
					// Now see if we've got a '\n' (LF) character.
					bool lf_present = false;
					if(src->current() == '\n')
					{
						lf_present = true;
						src->next();
					}
					
					// CR, CR, LF -> one new line
					// CR, LF -> one new line
					// LF -> one new line
					// CR (without being followed by LF) -> as many new lines as there are CR's
					size_t num_newlines_needed = 1;
					if(!lf_present)
						num_newlines_needed = num_cr;
					
					// Generate the required number of new-lines in form the user wants.
					for(size_t i = 0; i < num_newlines_needed; ++i)
					{
						for(const char *p = targetNewline; *p != '\0'; ++p)
							doWrite(*p, fo);
					}
					
					continue;
				}
				
				doWrite(src->current(), fo);
				
				src->next();
			}
			
			// add error for version 1.01.
			if(src->current() != NCC_STATUS_EOS)
			{
				if(src->status().empty())
					std::cout << "ERROR: An unknown internal error has occurred" << std::endl;
				else
					std::cout << "ERROR: " << src->status() << std::endl;
				ok = false;
			}
		}
		catch(std::exception &e)
		{
			if(freeSrc != NULL)
			{
				delete freeSrc;
				freeSrc = NULL;
			}
			throw e;
		}
		
		if(freeSrc != NULL)
		{
			delete freeSrc;
			freeSrc = NULL;
		}
		
		return ok;
	}

private:
	// Note: it would be best if we checked that the fputc()'s used here
	// actually succeeded in writing. It is safe to throw an exception if
	// the write fails.
	void doWrite(S4 chr, FILE *fo)
	{
		if(out_utf_bits == 7)
		{
			if(chr >= 0 && chr <= 127)
				fputc(chr, fo);
			else
			if(!failOnInvalidChars)
				fputc('?', fo);
			else
			if(chr >= 128 && chr <= 255)
			{
				std::stringstream ss;
				ss << std::hex << static_cast<U4>(chr) << std::dec;
				throw std::runtime_error(std::string("attempted to generate character 0x") + ss.str() + std::string(" but output format is set to Standard ASCII"));
			}
			else
				throw std::runtime_error("Internal UNICODE conversion error (1)");
		}
		else
		if(out_utf_bits == 8)
		{
			if(chr >= 0 && chr <= 255)
				fputc(chr, fo);
			else
				throw std::runtime_error("Internal UNICODE conversion error (2)");
		}
		else
		if(out_utf_bits == 16)
		{
			if(chr >= 0 && chr <= 65535)
			{
				if(out_big_endian)
				{
					fputc((chr >> 8) & 0xff, fo);
					fputc((chr >> 0) & 0xff, fo);
				}
				else
				{
					fputc((chr >> 0) & 0xff, fo);
					fputc((chr >> 8) & 0xff, fo);
				}
			}
			else
				throw std::runtime_error("Internal UNICODE conversion error (3)");
		}
		else
			throw std::runtime_error("Internal UNICODE conversion error (4)");
	}

	void write_bom(FILE *fo)
	{
		if(out_want_bom)
		{
			if(out_utf_bits == 8)
			{
				// Write UTF-8 Byte Order Mark.
				fputc(0xef, fo);
				fputc(0xbb, fo);
				fputc(0xbf, fo);
			}
			else
			if(out_utf_bits == 16)
			{
				if(out_big_endian)
				{
					// Write UTF-16-BE Byte Order Mark.
					fputc(0xfe, fo);
					fputc(0xff, fo);
				}
				else
				{
					// Write UTF-16-LE Byte Order Mark.
					fputc(0xff, fo);
					fputc(0xfe, fo);
				}
			}
		}
	}
};

}	// namespace WsuInternal

using WsuInternal::WsuConverter;

int main(int argc, const char **argv)
{
	using namespace std;
	std::string ofilename;
	FILE *fi = NULL;
	FILE *fo = NULL;
	try
	{
		using namespace NetwideCompilerCollection;
		
		std::list<std::string> cmdLine;
		NccCommonUtils::getCommandLine(cmdLine, argc, argv);
		
		for(std::list<std::string>::iterator i = cmdLine.begin(); i != cmdLine.end(); ++i)
		{
			if(*i == "--about")
			{
				print_about();
				return 0;
			}
		}
		
		for(std::list<std::string>::iterator i = cmdLine.begin(); i != cmdLine.end(); ++i)
		{
			if(*i == "--help")
			{
				print_help();
				return 0;
			}
		}
		
		std::list<std::string> options;
		std::list<std::string> files;
		if(!cmdLine.empty())
		{
			std::list<std::string>::iterator i = cmdLine.begin();
			++i;	// skip program name (0th argument)
			for(; i != cmdLine.end(); ++i)
			{
				if(i->empty())
					continue;
				if((*i)[0] == '-')
					options.push_back(*i);
				else
					files.push_back(*i);
			}
		}
		
		// Parse options.
		WsuConverter converter;
		if(!converter.parseOptions(options))
		{
			return 1;
		}

		if(files.size() != 2)
		{
			print_banner();
			print_basic_copyright();
			std::cout << std::endl;
			print_usage();
			std::cout << "For a list of options, use the --help option." << std::endl;
			std::cout << std::endl;
			std::cout << "ERROR: Missing input and/or output file name." << std::endl;
			return 1;
		}
		
		// Make sure we can open in file before we do anything else.
		std::string infilename = files.front();
		fi = NccCommonUtils::universalOpen(infilename.c_str(), "rb");
		if(fi == NULL)
		{
			std::cout << "ERROR: Unable to open file for reading: " << infilename << std::endl;
			return 1;
		}

		// Overwrite (or create) output file.
		ofilename = files.back();
		fo = NccCommonUtils::universalOpen(ofilename.c_str(), "wb");
		
		if(fo == NULL)
		{
			fclose(fi);
			std::cout << "ERROR: Unable to open file for writing: " << ofilename << std::endl;
			return 1;
		}
		
		converter.convert(fi, fo);
	}
	catch(std::exception &e)
	{
		// this should also catch bad alloc's
		std::cout << "fatal error: " << e.what() << std::endl;
		if(fi != NULL)
			fclose(fi);
		if(fo != NULL)
			fclose(fo);
		if(!ofilename.empty())
			NetwideCompilerCollection::NccCommonUtils::removeFile(ofilename.c_str());
		return 1;
	}
	catch(...)
	{
		std::cout << "fatal: unknown internal error" << std::endl;
		if(fi != NULL)
			fclose(fi);
		if(fo != NULL)
			fclose(fo);
		if(!ofilename.empty())
			NetwideCompilerCollection::NccCommonUtils::removeFile(ofilename.c_str());
		return 1;
	}
	
	if(fi != NULL)
		fclose(fi);
	if(fo != NULL)
		fclose(fo);

	return 0;
}
